{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2024-08-01T18:42:51.494276Z",
     "start_time": "2024-08-01T18:42:20.623296Z"
    }
   },
   "outputs": [],
   "source": [
    "## Please replace all path to your path before stating to evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import os\n",
    "import autogen\n",
    "from autogen.coding import LocalCommandLineCodeExecutor\n",
    "from autogen import AssistantAgent, UserProxyAgent\n",
    "from IPython.display import Image, display\n",
    "# import fitz  # PyMuPDF\n",
    "import json\n",
    "import base64\n",
    "import re\n",
    "import time\n",
    "import pandas as pd\n",
    "from tqdm.notebook import tqdm\n",
    "your_api_key = 'your-api'\n"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "49141aad778d65be"
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "os.environ[\"OPENAI_API_KEY\"]  = your_api_key"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-08-01T18:42:51.505755Z",
     "start_time": "2024-08-01T18:42:51.494789Z"
    }
   },
   "id": "712723600688536c",
   "execution_count": 2
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "MODEL_LIMITS = {\n",
    "    \"gpt-3.5-turbo-0125\": 16_385,\n",
    "    \"gpt-4-turbo-2024-04-09\": 128_000,\n",
    "    \"gpt-4o-2024-05-13\": 128_000,\n",
    "    \"gpt-4o-mini-2024-07-18\": 128_000\n",
    "}\n",
    "\n",
    "# The cost per token for each model input.\n",
    "MODEL_COST_PER_INPUT = {\n",
    "    \"gpt-3.5-turbo-0125\": 0.0000005,\n",
    "    \"gpt-4-turbo-2024-04-09\": 0.00001,\n",
    "    \"gpt-4o-2024-05-13\": 0.000005,\n",
    "    \"gpt-4o-mini-2024-07-18\": 0.00000015\n",
    "}\n",
    "\n",
    "# The cost per token for each model output.\n",
    "MODEL_COST_PER_OUTPUT = {\n",
    "    \"gpt-3.5-turbo-0125\": 0.0000015,\n",
    "    \"gpt-4-turbo-2024-04-09\": 0.00003,\n",
    "    \"gpt-4o-2024-05-13\": 0.000015,\n",
    "    \"gpt-4o-mini-2024-07-18\": 0.0000006\n",
    "}\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-08-01T20:57:02.127103Z",
     "start_time": "2024-08-01T20:57:01.948611Z"
    }
   },
   "id": "a6132e2e406890f6",
   "execution_count": 7
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "74"
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = []\n",
    "\n",
    "with open(\"./data.json\", \"r\") as f:\n",
    "    for line in f:\n",
    "        data.append(eval(line))\n",
    "        \n",
    "len(data)"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-08-01T18:42:51.529153Z",
     "start_time": "2024-08-01T18:42:51.511326Z"
    }
   },
   "id": "fdb261df9c451d6a",
   "execution_count": 4
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "\n",
    "config_list = autogen.config_list_from_json(\n",
    "    \"OAI_CONFIG_LIST\",\n",
    "    filter_dict={\"tags\": [\"gpt-4\"]},  # comment out to get all\n",
    ")\n",
    "\n",
    "# create an AssistantAgent named \"assistant\"\n",
    "assistant = autogen.AssistantAgent(\n",
    "    name=\"assistant\",\n",
    "    llm_config={\n",
    "        \"cache_seed\": 41,  # seed for caching and reproducibility\n",
    "        \"config_list\": config_list,  # a list of OpenAI API configurations\n",
    "        \"temperature\": 0,  # temperature for sampling\n",
    "    },  # configuration for autogen's enhanced inference API which is compatible with OpenAI API\n",
    ")\n",
    "\n",
    "# create a UserProxyAgent instance named \"user_proxy\"\n",
    "user_proxy = autogen.UserProxyAgent(\n",
    "    name=\"user_proxy\",\n",
    "    human_input_mode=\"NEVER\",\n",
    "    max_consecutive_auto_reply=10,\n",
    "    is_termination_msg=lambda x: x.get(\"content\", \"\").rstrip().endswith(\"TERMINATE\"),\n",
    "    code_execution_config={\n",
    "        # the executor to run the generated code\n",
    "        \"executor\": LocalCommandLineCodeExecutor(work_dir=\"coding\"),\n",
    "    },\n",
    ")\n",
    "start = time.time()\n",
    "# the assistant receives a message from the user_proxy, which contains the task description\n",
    "chat_res = user_proxy.initiate_chat(\n",
    "    assistant,\n",
    "    message=\"\"\"What date is today? Compare the year-to-date gain for META and TESLA.\"\"\",\n",
    "    summary_method=\"reflection_with_llm\",\n",
    ")\n",
    "consume = time.time() - start"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "c2e9b8045a2c9400"
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "chat_res"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "95e92fb802965df5",
   "execution_count": null
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "model = \"gpt-3.5-turbo-0125\"\n",
    "prompt_tokens = chat_res.cost['usage_including_cached_inference'][model]['prompt_tokens']\n",
    "completion_tokens = chat_res.cost['usage_including_cached_inference'][model]['completion_tokens']\n",
    "cost = chat_res.cost['usage_including_cached_inference'][model]['cost']\n",
    "summary = chat_res.summary\n",
    "history = chat_res.chat_history                "
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-07-16T04:51:18.498932Z",
     "start_time": "2024-07-16T04:51:18.471010Z"
    }
   },
   "id": "9e31c75f84e55e65",
   "execution_count": 26
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "{'total_cost': 0.0007615,\n 'gpt-3.5-turbo-0125': {'cost': 0.0007615,\n  'prompt_tokens': 752,\n  'completion_tokens': 257,\n  'total_tokens': 1009}}"
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chat_res.cost['usage_including_cached_inference']"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-07-16T04:34:09.351010Z",
     "start_time": "2024-07-16T04:34:09.292337Z"
    }
   },
   "id": "c254010f687d73b0",
   "execution_count": 15
  },
  {
   "cell_type": "code",
   "outputs": [
    {
     "data": {
      "text/plain": "0.039936065673828125"
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "consume"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-07-16T00:13:38.185420Z",
     "start_time": "2024-07-16T00:13:38.178050Z"
    }
   },
   "id": "c2b1a688821311f5",
   "execution_count": 11
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "def get_response(text, config_list):\n",
    "    assistant = autogen.AssistantAgent(\n",
    "        name=\"assistant\",\n",
    "        llm_config={\n",
    "            \"cache_seed\": 41,  # seed for caching and reproducibility\n",
    "            \"config_list\": config_list,  # a list of OpenAI API configurations\n",
    "            \"temperature\": 0,  # temperature for sampling\n",
    "        },  # configuration for autogen's enhanced inference API which is compatible with OpenAI API\n",
    "    )\n",
    "    \n",
    "    # create a UserProxyAgent instance named \"user_proxy\"\n",
    "    user_proxy = autogen.UserProxyAgent(\n",
    "        name=\"user_proxy\",\n",
    "        human_input_mode=\"NEVER\",\n",
    "        max_consecutive_auto_reply=10,\n",
    "        is_termination_msg=lambda x: x.get(\"content\", \"\").rstrip().endswith(\"TERMINATE\"),\n",
    "        code_execution_config={\n",
    "            # the executor to run the generated code\n",
    "            \"executor\": LocalCommandLineCodeExecutor(work_dir=\"coding\"),\n",
    "        },\n",
    "    )\n",
    "    # the assistant receives a message from the user_proxy, which contains the task description\n",
    "    chat_res = user_proxy.initiate_chat(\n",
    "            assistant,\n",
    "            message=text,\n",
    "            summary_method=\"reflection_with_llm\",\n",
    "        )\n",
    "    return chat_res\n",
    "    "
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-08-01T18:42:51.550245Z",
     "start_time": "2024-08-01T18:42:51.531874Z"
    }
   },
   "id": "478879b48a8e137b",
   "execution_count": 5
  },
  {
   "cell_type": "code",
   "outputs": [],
   "source": [
    "# model = \"gpt-3.5-turbo-0125\"\n",
    "# model = \"gpt-4o-2024-05-13\"\n",
    "model = \"gpt-4-turbo-2024-04-09\"\n",
    "config_list = [{\"model\": model, \"api_key\": your_api_key}]\n",
    "total_cost = 0\n",
    "\n",
    "instruction = \"You are a data scientist. I have a data modeling task. You must give me the predicted results as a CSV file as detailed in the following content. You should try your best to predict the answer. I provide you with three files. One is training data, one is test data. There is also a sample file for submission\"\n",
    "\n",
    "save_path = \"./output_model/\"\n",
    "\n",
    "data_path = \"./data_filted_csv/data_resplit/{name}/\" ## replace this to your data file\n",
    "\n",
    "for id in tqdm(range(0, len(data))):\n",
    "# for id in tqdm([0]):\n",
    "    # print(sample)\n",
    "    name = data[id]['name']\n",
    "    with open(f\"./data_filted_csv/cleaned_task/{name}.txt\", \"r\") as f:\n",
    "        description = f.read()\n",
    "    \n",
    "    text = (f\"\\n \\n All three data files can be found in the folder: {data_path}. After the data modeling, please give me the prediction resutls for the test file. You must\"\n",
    "            f\" save the answer as a csv file. I won't run your code and you must run the code for the predicted results and give the submission file. The file should be saved in the path ./output_model/{model}-autoagent/{name}.csv\") \n",
    "    \n",
    "    all_context = instruction + \"\\n\" + description + \"\\n\" + text\n",
    "    input_t = all_context\n",
    "\n",
    "    # input_t = truncate_text(all_context, 2000)\n",
    "    start = time.time()\n",
    "    cost = 0\n",
    "    error = \"\"\n",
    "    prompt_tokens = completion_tokens = 0\n",
    "    try:\n",
    "        response = get_response(input_t, config_list)\n",
    "        prompt_tokens = response.cost['usage_including_cached_inference'][model]['prompt_tokens']\n",
    "        completion_tokens = response.cost['usage_including_cached_inference'][model]['completion_tokens']\n",
    "        cost = response.cost['usage_including_cached_inference'][model]['cost']\n",
    "        summary = response.summary\n",
    "        history = response.chat_history\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        # time.sleep(3)\n",
    "        error = str(e)\n",
    "        # cost = 0\n",
    "        history = \"I cannot solve this task.\"\n",
    "        summary = \"I cannot solve this task.\"\n",
    "        print(history)\n",
    "        print(e)\n",
    "        time.sleep(3)\n",
    "                # all_mess.append(\"I cannot solve this task.\")\n",
    "    total_cost += cost\n",
    "    print(\"Total cost: \", total_cost)\n",
    "\n",
    "    if not os.path.exists(f\"{save_path}{model}-autoagent/\"):\n",
    "        os.makedirs(f\"{save_path}{model}-autoagent/\")\n",
    "    with open(f\"{save_path}{model}-autoagent/{name}.json\", \"w\") as f:\n",
    "        json.dump({\"name\": name, \"model\": model, \"input\": prompt_tokens,\n",
    "                            \"output\": completion_tokens, \"cost\": cost, \"time\": time.time()-start, \"error\": error, 'summary': summary, \"history\": history}, f)\n",
    "    \n",
    "\n",
    "    # if total_cost > 100:\n",
    "    #     break"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "eee15c339d6795b5",
   "execution_count": null
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
